#!/usr/bin/python3
# -*- coding:utf-8 -*-
# project:
# user:86175
# Author: 亿只羊
# createtime: 2022/5/17 17:29

"""
aclimdb数据集预处理
"""
from keras.preprocessing import sequence
from keras.preprocessing.text import Tokenizer
import os
import re
from sklearn.ensemble import RandomForestClassifier


class imdbRfc:
    def rm_tags(text):
        re_tag = re.compile(r'<[^>]+>')
        return re_tag.sub('', text)


    def read_files(filetype):
        path = "data/aclImdb/"
        file_list = []

        positive_path = path + filetype + "/pos/"
        for f in os.listdir(positive_path):
            file_list += [positive_path + f]

        negative_path = path + filetype + "/neg/"
        for f in os.listdir(negative_path):
            file_list += [negative_path + f]

        print('read', filetype, 'files:', len(file_list))

        all_labels = ([1] * 12500 + [0] * 12500)

        all_texts = []
        for fi in file_list:
            with open(fi, encoding='utf8') as file_input:
                all_texts += [rm_tags(" ".join(file_input.readlines()))]

        return all_labels, all_texts
    y_train, train_text = read_files("train")
    y_test, test_text = read_files("train")

    token = Tokenizer(num_words=2000)
    token.fit_on_texts(train_text)
    x_train_seq = token.texts_to_sequences(train_text)
    x_test_seq = token.texts_to_sequences(test_text)
    x_train = sequence.pad_sequences(x_train_seq, maxlen=100)
    x_test = sequence.pad_sequences(x_test_seq, maxlen=100)




#型
model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
model.fit(X, y)
predictions = model.predict(X_test)

output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
output.to_csv('submissions/my_submission.csv', index=False)
